package com.g2room.driving.grap.images;

import java.io.IOException;
import java.net.MalformedURLException;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.eclipse.jetty.util.StringUtil;

import com.g2room.driving.grap.GenericGrap;
import com.g2room.driving.model.po.Image;
import com.g2room.driving.model.po.Theme;
import com.g2room.driving.util.DbUtil;
import com.g2room.driving.util.ImageUtil;
import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlAnchor;
import com.gargoylesoftware.htmlunit.html.HtmlDefinitionDescription;
import com.gargoylesoftware.htmlunit.html.HtmlImage;
import com.gargoylesoftware.htmlunit.html.HtmlItalic;
import com.gargoylesoftware.htmlunit.html.HtmlListItem;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import com.gargoylesoftware.htmlunit.html.HtmlParagraph;

public class GrapDu114DataByProxy {
	static Connection connection = null;
	static PreparedStatement  ps = null;
	static Statement  statement = null;
	
	static Map<String, String> errorUrls = new HashMap<String, String>();
	private static final String dir = "E:/高绪国/资料/meinv/du114/";
	private static final String source = "114图库";
	
	public static void main(String[] args) throws Exception{
		
		GrapDu114DataByProxy gkd = new GrapDu114DataByProxy();
		gkd.updateDesc();
//		gkd.getPage();
//		insertError2Db();
//		DbUtil.closeConn();
    }
	
	public void updateDesc() {
		Map<Integer, String> themes = getAllTheme();
		WebClient wc = GenericGrap.getWebClient();
		for(Integer id : themes.keySet()) {
			String url = themes.get(id);
			HtmlPage page = null;
	        try {
	        	page = wc.getPage(url);
	        } catch (Exception e) {
	        	errorUrls.put(url, "id"+id);
	        	e.printStackTrace();
	        	return;
	        }
	        
	        // 获取描述 descriptionBox
	        List<?> descList = page.getByXPath("//div[@class=\"descriptionBox\"]/p");
	        String desc = "";
	        for(Object o : descList) {
	        	desc = ((HtmlParagraph) o).asText();
	        }
	        
	        if(StringUtil.isBlank(desc)) continue;
	        // 
	        try {
		        String sql = "update i_theme set description=? where id=?";
		        System.out.println("------=" + sql);
	            if(sqlCount == 0) {
	            	ps = DbUtil.getConn().prepareStatement(sql);
	            	sqlCount++;
	            }
				ps.setString(1, desc);
				ps.setInt(2, id);
				ps.execute();
			} catch(Exception e) {
				e.printStackTrace();
			}
		}
		if (null != wc) wc.close();
	}
	
	/**
	 * 获取分页数据
	 * @param url
	 * @throws FailingHttpStatusCodeException
	 * @throws MalformedURLException
	 * @throws IOException
	 */
	public void getPage() throws FailingHttpStatusCodeException, MalformedURLException, IOException {
		int pageNum = 0;

		String categoryName="人体艺术", pageUrl ="http://www.du114.com/meinvtupian/rentiyishu/list_75_##.html";
		pageNum = 27;
//		String categoryName="美女写真", pageUrl ="http://www.du114.com/meinvtupian/meinvxz/list_35_##.html";
//		pageNum = 28;
//		String categoryName="内衣美女", pageUrl ="http://www.du114.com/meinvtupian/nayimeinv/list_45_##.html";
//		pageNum = 25;
    	List<String> urls = getUrlTheme();
    	
		List<String> pages = new ArrayList<String>();
		for(int i = 1; i<=pageNum; i++) {
			pages.add(pageUrl.replace("##", String.valueOf(i)));
		}
		for(String url : pages) {
			System.out.println("--------------start get page data!" + categoryName + ",url=" + url);
			WebClient wc = GenericGrap.getWebClient();
			//获取页面
			HtmlPage page = null;
	        try {
	        	page = wc.getPage(url);
	        } catch (Exception e) {
	        	errorUrls.put(url, categoryName);
	        	e.printStackTrace();
	        	return;
	        }
	        
			// 获取每页的图片列表
			List<?> list = page.getByXPath("//div[@id=\"imgList\"]/ul/li");
			List<Theme> themes = new ArrayList<Theme>();
			for(Object hs : list) {
				HtmlListItem listItem = (HtmlListItem) hs;
				Theme theme = new Theme();
				List<?> alist = listItem.getByXPath("a[@class=\"picLink\"]");
				for(Object o : alist) {
					theme.setCategoryName(categoryName);
					// 获取详细页面链接
					HtmlAnchor detailAnchor = (HtmlAnchor) o;
					// 获取封面图片
					List<?> imgList = detailAnchor.getElementsByTagName("img");
					if(null == imgList || imgList.isEmpty()) continue;

					if(urls.contains(detailAnchor.getHrefAttribute())) {
			    		System.out.println("url 存在==" + detailAnchor.getHrefAttribute());
			    		continue;
			    	}
					
					theme.setCoverOriginal(((HtmlImage)imgList.get(0)).getSrcAttribute());
					theme.setSourceUrl(detailAnchor.getHrefAttribute());
					theme.setTitle(detailAnchor.getAttribute("title"));
				}
				// 获取图片数和发布时间
				List<?> ilist = listItem.getByXPath("span[@class=\"info\"]/i");
				for(Object o : ilist) {
					// 获取图片数和发布时间
					HtmlItalic italic = (HtmlItalic) o;
					String text = italic.asText();
					if(StringUtil.isBlank(text)) continue;
					if(text.endsWith("P")) {
						theme.setImgNum(Integer.valueOf(text.replace("P", "")));
					} else {
						theme.setPublishTime(text);
					}
				}
				
				themes.add(theme);
			}
	        if (null != wc) wc.close();
	        
	        for(Theme theme : themes) {
	        	getDetail(theme);
	        }
		}
	}
	
	public void getDetail(Theme theme) throws FailingHttpStatusCodeException, MalformedURLException, IOException {
		System.out.println("*************start get detail data!  " + theme.getSourceUrl());
		WebClient wc = GenericGrap.getWebClient();
		HtmlPage page = null;
        try {
        	page = wc.getPage(theme.getSourceUrl());
        } catch (Exception e) {
        	errorUrls.put(theme.getSourceUrl(), theme.getCategoryName());
        	e.printStackTrace();
        	return;
        }
        
        // 获取Tags
        List<?> ddList = page.getByXPath("//div[@id=\"articleTag\"]/dl/dd");
        String tags = "";
        for(Object o : ddList) {
        	HtmlDefinitionDescription dd = (HtmlDefinitionDescription) o;
        	tags += dd.asText().trim() + "|";
        }
        theme.setTags(tags);
        
        // 获取描述 descriptionBox
        List<?> descList = page.getByXPath("//div[@class=\"descriptionBox\"]/p");
        for(Object o : descList) {
        	theme.setDescription(((HtmlParagraph) o).asText());
        }
        if (null != wc) wc.close();
        String filePath = "";
		try {
			filePath = ImageUtil.downloadImage(theme.getCoverOriginal(), dir, "封面/" + theme.getCategoryName(), null);
			theme.setCover(filePath);
		} catch (Exception e) {
			errorUrls.put(theme.getCoverOriginal(), theme.getCategoryName()+"封面图片下载失败");
			e.printStackTrace();
		}
        // 插入数据库表
        theme.setId(inserTheme(theme));
        

        // 获取详细页面图片
        String imgPagrUrl;
        for(int i = 1; i< theme.getImgNum(); i++) {
        	if(i == 1) {
        		imgPagrUrl = theme.getSourceUrl();
        	} else {
        		imgPagrUrl = theme.getSourceUrl().replace(".html", "_"+i+".html");
        	}
        	
        	getImgs(imgPagrUrl, theme.getId(), theme.getCategoryName());
        }
	}
	
	public void getImgs(String imgUrl, int themeId, String categoryName) throws FailingHttpStatusCodeException, MalformedURLException, IOException {
		System.out.println("---------start get detail data!  " + imgUrl);
		WebClient wc = GenericGrap.getWebClient();
		HtmlPage page = null;
        try {
        	page = wc.getPage(imgUrl);
        } catch (Exception e) {
        	errorUrls.put(imgUrl, categoryName);
        	e.printStackTrace();
        	return;
        }
        

        Image image = new Image();
        image.setThemeId(themeId);
		// 获取分页数据  img onload="btnaddress(1);"
        List<?> imgList = page.getByXPath("//div[@id=\"picBody\"]/p/a/img");
        if(null == imgList || imgList.isEmpty()) return;
        String filePath = "";
        for(Object o : imgList) {
			HtmlImage img = (HtmlImage) o;
			image.setImageOriginal(img.getSrcAttribute());
			if (null != wc) wc.close();
			try {
				filePath = ImageUtil.downloadImage(image.getImageOriginal(), dir, categoryName, null);
				image.setImage(filePath);
			} catch (Exception e) {
				errorUrls.put(image.getImageOriginal(), source + categoryName);
				e.printStackTrace();
				continue;
			}
			// 插入数据库
			inserImage(image);
			// 错误数据每50条插入一次
			if(errorUrls.size() >50) {
				insertError2Db();
				errorUrls.clear();
			}
        }
	}
	
	


	int sqlCount = 0;
	private int inserTheme(Theme theme) {
		try {
            String sql = "insert into i_theme(title,cover,category_name,publish_time,cover_original,tags,source_url,source,description) values(?,?,?,?,?,?,?,?,?)";
            ps = DbUtil.getConn().prepareStatement(sql,Statement.RETURN_GENERATED_KEYS);  
            if(sqlCount == 0) {
            	ps = DbUtil.getConn().prepareStatement(sql);
            	sqlCount++;
            }
			ps.setString(1, theme.getTitle());
			ps.setString(2, theme.getCover());
			ps.setString(3, theme.getCategoryName());
			ps.setString(4, theme.getPublishTime());
			ps.setString(5, theme.getCoverOriginal());
			ps.setString(6, theme.getTags());
			ps.setString(7, theme.getSourceUrl());
			ps.setString(8, source);
			ps.setString(9, theme.getDescription());
			ps.execute();
		} catch(Exception e) {
			e.printStackTrace();
		}
		return getId();
	}

	private List<String> getUrlTheme() {
		List<String> urls = new ArrayList<String>();
		try {
            String sql = "select source_url from i_theme";
            if(statemenetCount==0) statement = DbUtil.getConn().createStatement();  
            ResultSet rs = statement.executeQuery(sql);
            while(rs.next()) {
            	urls.add(rs.getString("source_url"));
            }
            rs.close();
		} catch(Exception e) {
			e.printStackTrace();
		}
		return urls;
	}
	int statemenetCount = 0;
	private int getId() {
		int id = 0;
		try {
            String sql = "select max(id) as id from i_theme";
            if(statemenetCount==0)  statement = DbUtil.getConn().createStatement();  
            ResultSet rs = statement.executeQuery(sql);
            while(rs.next()) {
            	id = rs.getInt("id");
            }
            rs.close();
		} catch(Exception e) {
			e.printStackTrace();
		}
		return id;
	}


	int sqlImageCount = 0;
	private void inserImage(Image image) {
		try {
            String sql = "insert into i_image(theme_id,image,image_original) values(?,?,?)";
            ps = DbUtil.getConn().prepareStatement(sql,Statement.RETURN_GENERATED_KEYS);  
            if(sqlImageCount == 0) {
            	ps = DbUtil.getConn().prepareStatement(sql);
            	sqlImageCount++;
            }
			ps.setInt(1, image.getThemeId());
			ps.setString(2, image.getImage());
			ps.setString(3, image.getImageOriginal());
			ps.execute();
		} catch(Exception e) {
			e.printStackTrace();
		}
	}
	
	public static void insertError2Db() {
		try {
			PreparedStatement  ps = DbUtil.getConn().prepareStatement("insert into i_grap_err(title,source_url,source) values(?,?,?)");
			System.out.println("===================error===========================");
			for(String url : errorUrls.keySet()) {
				System.out.println(url + " --------------- " + errorUrls.get(url));
				ps.setString(1, errorUrls.get(url));
				ps.setString(2, url);
				ps.setString(3, source);
				ps.addBatch();
			}
			ps.executeBatch();
			if(null != ps) ps.close();
			if(null != connection) connection.close();
		} catch(Exception e) {
			e.printStackTrace();
		}
	}
	


	private Map<Integer, String> getAllTheme() {
		Map<Integer, String> urls = new HashMap<Integer, String>();
		try {
            String sql = "select id,source_url from i_theme where source='114图库' and description is null";
            if(statemenetCount==0) statement = DbUtil.getConn().createStatement();  
            ResultSet rs = statement.executeQuery(sql);
            while(rs.next()) {
            	urls.put(rs.getInt("id"), rs.getString("source_url"));
            }
            rs.close();
		} catch(Exception e) {
			e.printStackTrace();
		}
		return urls;
	}
}
